import time
from setting import *
# from tools import *
from lxml import html
from util.data_utils import *
from util.time_utils import *
from util.request_utils import *
import time
import random
import json

def get_url_data(key_url, group_name):
    print(getCurTime(), f"【{group_name} 开始】")

    response = ownRequest2(key_url)
    response_json = response.json()
    # 基本循环
    urls = {}
    for item in response_json:
        # 检查ID是否已存在于Redis中
        if not R.exists(item['id']):
            R.set(item['id'], str(time.time()))
            print(f"ID: {item['id']}")
            print(f"标题: {item['title']}")
            print(f"内容: {item['content']}")
            print(f"发布时间: {item['datetime']} {item['shorttime']}")
            print(f"分类: {item['catename']}")
            print(f"楼主: {item['louzhu']}")
            print(f"链接: {item['url']}")
            print("-" * 50)
            urls[item['id']] = f"{DOMAIN_NAME}{item['url']}"

    print("6666"); 
    print(urls)
    print("6666");

    if urls:   
        results = fet_article_details(urls, group_name)
        print(f"\n处理完成，共获取 {len(results)} 篇文章")
        
        for result in results:
            sleep_time = random.uniform(1, 2)
            time.sleep(sleep_time)
            result_json = json.dumps(result, ensure_ascii=False, indent=2)
            success = toDatabase(result)
            
            if success:
                print("数据推送成功")
            else:
                print("数据推送失败，从Redis中删除ID")
                # 推送失败时从Redis中删除对应的ID
                R.delete(result['id'])
                
            print(result_json)

    time.sleep(5)

def main():
    # 死循环
    while True:
        try:
            print(f"\n{getCurTime()} === 开始新一轮爬取 ===")
            
            # 创建并启动多个线程
            for items in key_url_lis:
                url = items[0]
                group_name = items[1]
                get_url_data(url, group_name)
                sleep_time = random.uniform(2, 5)
                print(f"等待 {sleep_time:.2f} 秒...")
                time.sleep(sleep_time)
            
            # 每轮结束后等待较长时间
            round_sleep = random.uniform(2, 3)  # 30-60秒
            print(f"{getCurTime()} 本轮结束，等待 {round_sleep:.2f} 秒后开始下一轮...")
            time.sleep(round_sleep)
            
        except KeyboardInterrupt:
            print(f"{getCurTime()} 用户中断程序")
            break
        except Exception as e:
            print(f"{getCurTime()} 发生错误: {e}, 10秒后重试...")
            time.sleep(5)

if __name__ == "__main__":
    # 买组  拼组
    key_url_lis = [
        ["https://news.ixbk.net/plus/json/push_22.json", "拼组"]
    ]
    
    main()
